# 爬取京东手机销量数据
# pyquery 选取 两个类  .gl-warp.clearfix 而不是  .gl-warp  clearfix
import random
import time

import requests
from pyquery import PyQuery as pq
from xlwt import Workbook

header = {
    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.9",
    "Accept-Encoding": "gzip, deflate",
    "Accept-Language": "zh-CN,zh;q=0.9",
    "Connection": "keep-alive",
    "Upgrade-Insecure-Requests": "1",
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/86.0.4240.198 Safari/537.36",
}

urls = [
    'https://search.jd.com/Search?keyword=%E6%89%8B%E6%9C%BA&stop=1&qrst=1&psort=3&vt=2&wq=%E6%89%8B%E6%9C%BA&psort=3&pvid=84d0841270334d4d803756f8fc0be0fd&cid3=655&cid2=653&page={}&s=61&click=0'.format(
        str(page)) for page in range(1, 8, 2)]


def getHtml(url):
    try:
        response = requests.get(url, headers=header)
        if response.status_code == requests.codes.ok:
            html = response.content.decode("utf-8")
            # with open("jd_mobile.html", "w", encoding="utf-8") as fp:
            #     fp.write(html)
            return html
        else:
            return None
    except Exception as e:
        return None


def parseHtml(html):
    doc = pq(html)
    lis = doc(".gl-warp.clearfix > li")
    for li in lis.items():
        if (len(li("div.p-name > a > em")) > 0):
            name = li("div.p-name > a > em").text().replace('\n', ' ').replace('\r', ' ')
        else:
            name = li(' div > div.p-name.p-name-type-2 > a').attr('title')
        yield {
            "name": name,
            "price": li("div.p-price i").text().strip(),
            "seller": li("div.p-shop > span > a").html()
        }


def exec():
    header = ['排名', '产品', '价格', '卖家']
    excel = Workbook(encoding="utf-8")
    sheet_all = excel.add_sheet("所有手机销售排行")
    sheet_apple = excel.add_sheet("苹果手机销售排行")
    sheet_huawei = excel.add_sheet("华为手机销售排行")
    sheet_xiaomi = excel.add_sheet("小米手机销售排行")
    for x, y in enumerate(header):
        sheet_all.write(0, x, y)
        sheet_apple.write(0, x, y)
        sheet_huawei.write(0, x, y)
        sheet_xiaomi.write(0, x, y)
    i = 1
    apple_i = 1
    huawei_i = 1
    xiaomi_i = 1
    for url in urls:
        html = getHtml(url)
        for mobile_info in parseHtml(html):
            sheet_all.write(i, 0, str(i))
            sheet_all.write(i, 1, mobile_info['name'])
            sheet_all.write(i, 2, mobile_info['price'])
            sheet_all.write(i, 3, mobile_info['seller'])
            i += 1
            if mobile_info['name'].lower().find("apple") != -1 or mobile_info['name'].lower().find("苹果") != -1:
                sheet_apple.write(apple_i, 0, str(i))
                sheet_apple.write(apple_i, 1, mobile_info['name'])
                sheet_apple.write(apple_i, 2, mobile_info['price'])
                sheet_apple.write(apple_i, 3, mobile_info['seller'])
                apple_i += 1
            if mobile_info['name'].lower().find("华为") != -1 or mobile_info['name'].lower().find("huawei") != -1:
                sheet_huawei.write(huawei_i, 0, str(i))
                sheet_huawei.write(huawei_i, 1, mobile_info['name'])
                sheet_huawei.write(huawei_i, 2, mobile_info['price'])
                sheet_huawei.write(huawei_i, 3, mobile_info['seller'])
                huawei_i += 1
            if mobile_info['name'].lower().find("小米") != -1 or mobile_info['name'].lower().find("xiaomi") != -1:
                sheet_xiaomi.write(xiaomi_i, 0, str(i))
                sheet_xiaomi.write(xiaomi_i, 1, mobile_info['name'])
                sheet_xiaomi.write(xiaomi_i, 2, mobile_info['price'])
                sheet_xiaomi.write(xiaomi_i, 3, mobile_info['seller'])
                xiaomi_i += 1
        time.sleep(random.randint(0, 3))
    excel.save("手机销售排行.xls")


exec()
